<?php
	require("Util.php");
	include("PorterStemmer.php");
	
	$stat = buildSynonym();
	echo $stat;
	
	/*
		Splitting text for the wikification
	*/
	function buildSynonym () {
	   $folder_name = $_POST["templateFile"];
	   $folderpath = $GLOBALS['uploaddir']."/".$folder_name;
	   $input_file = $folderpath."/".$GLOBALS["wikibook"];//path for the wikified file
	  
	   echo "convert wiki to html<br />";
	   convertWikiToHTMLTag ($folderpath, $input_file);
	   echo "tidying text <br />";
	   tidyText($folderpath, $input_file);
     
	   return "&msg=Cleaning temporary links&result=no error"; 
	}
	 
	/*
		Convert each wiki tag to hyperlink tag.
		@param string $folder_name the path to the book directory
		@param string $input_file the path to the input XHTML file
	*/
	function convertWikiToHTMLTag ($folder_name, $input_file) {$stime = microtime(); $time1 = now($stime);
		echo "open redirect <br />";
		$index_list = openRedirects($folder_name);
		$redirect_list = $index_list[0];
		$phrase_list = $index_list[1];
		$count_list = $index_list[2];
		$data = file_get_contents($input_file);
	
		//remove exisiting bob from the file first
		$bob_pos = stripos($data,"<bob>");
		if ($bob_pos !== false) {
			$ebob_pos = stripos($data,"</bob>");
			$data = substr($data, 0, $bob_pos).substr($data,$ebob_pos + 6);
		}
	
		//--we need to get all the num ids first before removing wikilink inside hyperlink
		preg_match_all('/\[\[([\d]+)\|([^\]]+)\]\]/',$data,$matches, PREG_SET_ORDER);
		//--we need to remove wiki link if the term is inside another hyperlink
		$lapos = stripos($data, "<a");
		while ($lapos !== false) {
			if (substr($data, $lapos + 2, 1) == " ") {
				$clpos = stripos($data, ">", $lapos + 3);
			} else if (substr($data, $lapos + 2, 1) == ">") {
				$clpos = $lapos + 2;
			} else {
				$clpos = -1;
			}
			if ($clpos > -1) {
				$rapos = stripos($data,"</a>",$clpos + 1);
				$tmp = substr($data, $clpos + 1, $rapos - $clpos - 1);
				preg_match_all('/(\[\[[\d]+\|([^\]]+)\]\])/',$tmp,$tmp_matches, PREG_SET_ORDER);
				foreach ($tmp_matches as $tmp_val) {
					$tmp = str_replace($tmp_val[1],$tmp_val[2],$tmp);
				}
				$data = substr_replace($data,$tmp,$clpos + 1, $rapos - $clpos - 1);
				$rapos = stripos($data,"</a>",$clpos + 1);
				$lapos = stripos($data,"<a", $rapos + 4);
			} else {
				$lapos = false;
				break;
			}
		}
		
		$tooltip[0] = "tooltip text";
		$linklist[0] = "link text";
		$textCap[0] = "link text correct capital";
		foreach ($matches as $val) {
			//--we then need to create the tooltip and href for each of the id
			$aID = $val[1];
			$aLink = $val[2];
		
			//if the capitalisation in the wikipedia is more than one
			//and the text does not have capitalisation, we think that 
			//the text has been incorrectly linked to the article 
			//so just ignore it
			if (!$textCap[$aLink]) {
				$textCap[$aLink] = isCorrectWikiLink($aID, $aLink);
				if (strcmp($textCap[$aLink],"false") != 0)
					echo $aLink."---".$textCap[$aLink]."<br />";
			}
				
			if (strcmp($textCap[$aLink],"false") != 0) {
				//increment the frequency this text occurs
				$waText = cleanIndexEntry($aLink);
				$wpText = $waText;
				if ($textCap[$aLink] == "person") {
					$waText = convertName($waText);
				}
				$kaText = strtolower($waText);                      
				if (!$count_list[$kaText])
					$count_list[$kaText] = 0;
				$count_list[$kaText] = $count_list[$kaText] + 1;
				
				if (!$linklist[$aLink]) {
					if (!$tooltip[$aID]) {
						$url = $GLOBALS["wikiaddr"]."task=define&getImages=true&id=".$aID;
						$xmlContent = file_get_contents($url);
						
						//save the xml to a file
						$xmlfname = $folder_name.'/xmls/'.$aID.'.xml';
						if (!file_exists($xmlfname)) {
							$fp = modifyFile($xmlfname,'w');
							if (!$fp) die ("cannot save the xml");
							fwrite($fp,$xmlContent);
							fclose($fp);
						}
						
						//get the wikipedia article title
						$aText = "";
						$wTitle = "";
						preg_match_all("/<DefinitionResponse[\s]+([^>]+)>/", $xmlContent, $tmp_xmatches, PREG_SET_ORDER);
						foreach ($tmp_xmatches as $xmatches) {
							preg_match_all("/title=\"([^\"]+)\"/", $xmatches[1], $tmp_tmatches, PREG_SET_ORDER);
							foreach ($tmp_tmatches as $tmatches) {
								$wTitle = cleanWikiTitle($tmatches[1]);
								$wText = cleanIndexEntry($tmatches[1]);
								$wnText = $wText;
								if ($textCap[$aLink] == "person") {
									$wText = convertName($wText);
								}
								$kText = strtolower($wText);
								$linklist["wid:".$aID] = $kText;
								$aText = "<a href=\"http://www.wikipedia.org/wiki/";
								$aText .= str_replace(" ","_",$tmatches[1]);
								$aText .= "\" title=\"wikiid:".$aID."\">";
								
								if (!$phrase_list[$kText]) {
									$phrase_list[$kText] = array();
								}
								if (!in_array("wikiid:".$wText, $phrase_list[$kText])){
									array_push($phrase_list[$kText], "wikiid:".$wText);   
								}
								break;
							}
							break;
						}
						
						//get the article image
						preg_match_all("/<Image[\s]+url=\"([^\"]+)\"[\s]*\/>/", $xmlContent, $tmp_imatches, PREG_SET_ORDER);
						foreach ($tmp_imatches as $imatches) {
							$imgfname = $folder_name.'/images/'.$aID.'.jpg';
							if (!file_exists($imgfname)) {
								$contents = url_get_contents($imatches[1]);
								$fp = modifyFile($imgfname, 'w');
								if (!$fp) die ("cannot save the image");
								fwrite($fp, $contents);
								fclose($fp);
							}
							
							break;
						}
						
						//get the synonyms from the redirect list
						$redirect_list = getRedirects ($linklist["wid:".$aID], $redirect_list, $aID);
						//if we have modified the wiki title, add it as a redirect
						if (strcmp($wTitle,$wText) != 0)
							$redirect_list = setTextAsRedirect ($linklist["wid:".$aID], $wTitle, $redirect_list,0);
						if (strcmp($wnText,$wText) != 0)
							$redirect_list = setTextAsRedirect ($linklist["wid:".$aID], $wTitle, $redirect_list,0);
						$tooltip[$aID] = $aText;
					}
					
					//set the current anchor text as synonyms
					if (!$linklist[$aLink]) {
						$redirect_list = setTextAsRedirect ($linklist["wid:".$aID], $waText, $redirect_list,1);
						if (strcmp($waText, $aLink) != 0)
							$redirect_list = setTextAsRedirect ($linklist["wid:".$aID], $aLink, $redirect_list,0);
						if (strcmp($waText, $wpText) != 0)
							$redirect_list = setTextAsRedirect ($linklist["wid:".$aID], $wpText, $redirect_list,0);
						if (!$phrase_list[$kaText]) {
							$phrase_list[$kaText] = array();
						}
						if (!in_array($waText, $phrase_list[$kaText])) {
							array_push($phrase_list[$kaText], $waText);       
						}
					}
					
					$linklist[$aLink] = $linklist["wid:".$aID];
					$aText = $tooltip[$aID];
				
					//--we then replace the [[]] with the appropriate a href value
					$data = str_replace("[[".$aID."|".$aLink."]]",$aText.$aLink."</a>",$data);
				}
			} else {
				//we replace the [[]] to unlink term
				$data = str_replace("[[".$aID."|".$aLink."]]",$aLink,$data); 
			}
		}
		
		echo "convert wiki to html tag<br />";
		$time2 = now($stime);timeLog("build synonym", $time2-$time1);
		
		echo "save redirect <br />";
		$index_list = array();		
		array_push($index_list,$redirect_list);
		array_push($index_list,$phrase_list);
		array_push($index_list,$count_list);
		$bob_data = saveRedirects ($folder_name, $index_list);
		$time3 = now($stime);timeLog("build bob", $time3-$time2);
		
		echo "save text file<br />";
		$data .= $bob_data;
		$fh = modifyFile($input_file, 'w') or die("can't open file");
		fwrite($fh, $data);
		fclose($fh);
		$time4 = now($stime);timeLog("save text file", $time4-$time3);
	}
	
	/*
		Get the content inside a tag (<tag></tag>) in the
		specified text
		@param string $tag the tag name
		@param string $text the text
		@return array the content of that tag
	 */
	function getContentInsideTag ($tag, $text) {
		$start_tag = "<".$tag.">";
		$end_tag = "</".$tag.">";
	
		$start_pos = stripos($text, $start_tag, $start_pos);
		$tag_content = array();
		while ($start_pos !== false) {
			$start_pos += strlen($start_tag);
			$end_pos = stripos($text, $end_tag, $start_pos);
			if ($end_pos !== false) {
				array_push($tag_content, substr($text, $start_pos, 
								   				$end_pos - $start_pos));
				$start_pos = stripos($text, $start_tag, 
									 $end_pos + strlen($end_tag));
			} else {
				$start_pos = false;
			}
		}
		return $tag_content;
	}
	 
	/*
		Open the redirect file and write it into a list
		@param string $folder_name the path to the book directory
		@return array [the redirect list, the phrase list]
	 */
	function openRedirects ($folder_name) {
		$syn_file = $folder_name.'/repository/redirect.xml';
		$index_list = array();		
		$redirect_list = array();
		$phrase_list = array();
		$count_list = array();
		array_push($index_list,$redirect_list);
		array_push($index_list,$phrase_list);
		array_push($index_list,$count_list);
		
		if (!file_exists($syn_file)) {
			return $index_list;
		}
		
		//get the list of redirects
		$xml = file_get_contents($syn_file);
		$redirects = getContentInsideTag("Redirect",$xml);
		for ($i = 0; $i < count($redirects); $i++) {
			$redirect = $redirects[$i];
			
			//get the key term
			$key = "";
			preg_match_all("/<Term[\s]+title=\"([^\"]+)\"[^>]+>/", $redirect, $tmp_tmatches, PREG_SET_ORDER);
			foreach ($tmp_tmatches as $tmatches) {
				$key = $tmatches[1];
			}
			
			//get the synonyms term
			preg_match_all("/<Synonym[\s]+title=\"([^\"]+)\"([^>]+)>/", $redirect, $tmp_smatches, PREG_SET_ORDER);
			foreach ($tmp_smatches as $smatches) {
				if (!$redirect_list[$key])
					$redirect_list[$key] = array();
				
				array_push($redirect_list[$key], $smatches[1]);
				
				$isWiki = stripos($smatches[1], "wikiid:");
				if (($isWiki !== false) || (strcmp($smatches[1], "rb:true") == 0)){
					//if it is the original wiki article or the word found in the book
					preg_match_all("/phrase=\"([^\"]+)\"[\s]+count=\"([^\"]+)\"/", $smatches[2], $tmp_pmatches, PREG_SET_ORDER);
					foreach ($tmp_pmatches as $pmatches) {
						if (!$phrase_list[$key])
							$phrase_list[$key] = array();
						
						$pterm = $pmatches[1];
						if ($isWiki !== false) {
							$pterm = "wikiid:".$pterm;
						} else {
							//get the number of times the term is written in the book
							$cterm = (int) $pmatches[2];
							$count_list[$key] = $cterm;
						}
						
						array_push($phrase_list[$key], $pterm);
					}
				} 
			}
		}
		
		$index_list = array();	
		array_push($index_list,$redirect_list);
		array_push($index_list,$phrase_list);
		array_push($index_list,$count_list);
		return $index_list;
	}
	
	/*
	 	Check whether we need to write the letter header in the index
		@param string $last_alp The last letter header in the index
		@param string $key The index entry
		@return array The string representation for the letter header
									and the new letter header
	*/
	function writeBoBLetterHeader ($last_alp, $key) {
		$txt = "";
		$firstChar = substr(strtoupper($key), 0, 1);
		if (strcmp($firstChar, $last_alp) != 0) {
				//write the first character in BoB
				$last_alp = $firstChar;
				$txt .= "<p align='left'></p>\n";
		}
		$ret_val = array($txt, $last_alp);
		return $ret_val;
	}
	
	/*
		The link representation of the specified index entry
		@param string $str The entry
		@param array  $phrase_list The list of term with the correct capitalisation
		@param array  $count_list The list of frequency a term occurs in the book
	*/
	function bobLink ($str, $phrase_list, $count_list) {
		$txt = "<a href='asfunction:_root.gotoAndSearchPage,".$str."'>";
		$txt .= applyCapital($phrase_list[$str][0], $count_list[$str]);
		$txt .= "</a>";
		return $txt;
	}
	
	/*
	 	The string representation of an entry in the Back of the Book index
		@param array  $key The list of entry
		@param array  $synonym The list of original wiki article
		@param string $last_alp The last alphabet in the index entry
		@param array  $phrase_list The list of term with the correct capitalisation
		@param array  $count_list The list of frequency a term occurs in the book
		@return array The string representation for the index entry
									and the new letter header
	 */
	function writeBoBEntry ($key, $synonym,$last_alp, $phrase_list,$count_list) {
			$h_val = writeBoBLetterHeader($last_alp,$key[0]);
			
			$txt = $h_val[0];
			$txt .= "<bobitem>\n<p align='left'>";
			if (count($synonym) == 0) {
				/*$txt .= "<a href='asfunction:_root.gotoAndSearchPage,".$key[0]."'>";
				$txt .= applyCapital($phrase_list[$key[0]][0], $count_list[$key[0]])."</a>";*/
				$txt .= bobLink($key[0], $phrase_list, $count_list);
				$txt .= "</p>\n";
			} else {
				for ($i = 0; $i < count($key); $i++) {
					if ($i > 0)
						$txt .= ", ";
					/*$txt .= "<a href='asfunction:_root.gotoAndSearchPage,".$key[$i]."'>";
					$txt .= applyCapital($phrase_list[$key[$i]][0], $count_list[$key[$i]]);*/
					$txt .= bobLink($key[$i], $phrase_list, $count_list);
					//$txt .= "</a>";
				}
				$txt .= ", <i>see</i> ";
				$syn_cnt = 0;
				for ($i = 0; $i < count($synonym); $i++) {
					if (stripos($synonym[$i],"rb:true") === false) {
						if ($syn_cnt > 0)
							$txt .= ", ";
						/*$txt .= "<a href='asfunction:_root.gotoAndSearchPage,".$synonym[$i]."'>";
						$txt .= applyCapital($phrase_list[$synonym[$i]][0], $count_list[$synonym[$i]])."</a>";*/
						$txt .= bobLink($synonym[$i], $phrase_list, $count_list);
						$syn_cnt++;
					}
				}
				$txt .= "</p>\n";
			}
			$txt .= "</bobitem>\n";
			
			return array($txt,$h_val[1]);
	}
	
	/*
		Get the wikipedia term from the synonym list
		@param array $syn_list the list of synonym
		@return string the wiki term
	*/
	function getWikitermFrom ($syn_list) {
		for ($i = 0; $i < count($syn_list); $i++) {
			$st = $syn_list[$i];
			if ((strcmp($st, "rb:true") != 0) &&
				(stripos($st, "wikiid:") === false)) {
				return $st;
			}
		}
		return "";
	}
	
	/*
		Check whether the term has different stemmed version to
		all the other key in the array, and the wiki term
		@param array $key_list the list of key
		@param string $wiki_term the wiki article
		@param string $term the new key
		@return boolean whether the key is already seen
	 */
	function isInStemArray ($key_list, $wiki_term, $term) {
		$term_stem = PorterStemmer::Stem($term);
		if (strlen($wiki_term) > 0) {
			$wiki_stem = PorterStemmer::Stem($wiki_term);
			if (strcmp($wiki_stem, $term_stem) == 0) {
				return true;
			}
		}
		for ($i = 0; $i < count($key_list); $i++) {
			$key_stem = PorterStemmer::Stem($key_list[$i]);
			if (strcmp($term_stem, $key_stem) == 0) {
				return true;
			}
		}
		
		return false;
	}
	
	/*
		Create the Back of the Book index
		@param array $redirect_list the list of redirect
		@param array $phrase_list the list of term with the correct capitalisation
		@param array $count_list the list of frequency a term occurs
		@return string The string for the back of the book
	 */
	function writeBoB ($redirect_list, $phrase_list, $count_list) {$stime = microtime(); $time1 = now($stime);
		$keys = array_keys($redirect_list);
		sort($keys);
		$key_list = array();//the list of key with the same synonyms
		$syn_list = array();//list of synonyms for the key_list
		$last_alp = "";//the first character of the last written entry
		$last_wiki_term = "";//the last wikipedia article written
		$store_wiki_term = "";//the last wiki term pointed to by key_list
		
		$txt = "<bob>\n";
		$txt .= "<h1 addnumber='false' addtitle='true'>Index</h1>\n";
		for ($i = 0; $i < count($keys); $i++) {
			$key = $keys[$i];
			sort($redirect_list[$key]);
			$vals = $redirect_list[$key];
			
			//need to see whether this is an wiki article
			//or the synonym list, and if the word is in the book
			$sim = false;
			$isinbook = false;
			if ((ord($key) >= ord("a")) && (ord($key) <= ord("z"))) {
			  //only check for term that starts with an alphabetic letter
			  for ($j = count($vals) - 1; $j >= 0; $j--) {
				  $term = $vals[$j];
				  if (stripos($term, "wikiid:") !== false) {
					  //if it is the original wiki article
					  $sim = true;
					  break;
				  } else if (strcmp($term, "rb:true") == 0) {
					  //if it is a word in the book
					  //check that it has different stemmed version
					  //than the wiki term
					  $new_wiki_term = getWikitermFrom($vals);
					  $tmp_arr = array();
					  if (isInStemArray ($tmp_arr, $new_wiki_term, $key) == false) {
					  	$isinbook = true;
					  }
				  }
			  }
			}
			
			if (($sim == true) || ($isinbook == true)) {
				if ($sim == true) {//if the term is the original wiki article
					if (count($key_list) > 0) {
						if ((strlen($store_wiki_term) == 0) ||
							((strcmp($store_wiki_term, $key) != 0) &&
							 (strcmp($store_wiki_term, $last_wiki_term) != 0))) {
						  //write the previous key list
						  //if the list wiki term is not equal ot the last wiki term
						  //and not equal to the soon to be written wiki term
						  $h_val = writeBoBEntry($key_list,$syn_list,$last_alp,$phrase_list,$count_list);
						  $txt .= $h_val[0];
						  $last_alp = $h_val[1];
						}
					}
					//write this term to the BoB as it's own entry
					$last_wiki_term = $key;
					$key_list = array();
					$syn_list = array();
					$store_wiki_term = "";
					$h_val = writeBoBEntry(array($key),$syn_list,$last_alp,$phrase_list,$count_list);
					$txt .= $h_val[0];
					$last_alp = $h_val[1];
				}
				
			  for ($j = 0; ($j < count($vals)) && ($sim == false); $j++) {
				  //need to see whether this entry is point to the
				  //same wiki article as the previous entry
				  $term = $vals[$j];
				  for ($k = 0; $k < count($syn_list); $k++) {
					if (strcmp($syn_list[$k], "rb:true") != 0) {
					  if (strcmp($term,$syn_list[$k]) == 0) {
						  if (isInStemArray ($key_list, $store_wiki_term, $key) == false) {
						  	//add this term to the previous entry
						  	array_push($key_list, $key);
						  	if (count($vals) != count($syn_list)) {
							 	 echo $key." count: ".count($syn_list)." but val: ".count($vals)."<br />";
						  	}
						  }
						  $sim = true;
						  break;
					  }
					}
				}
			  }
			  if ($sim == false) {
				  //the term point to a different wiki article
				  if (count($key_list) > 0) {
					  if ((strlen($store_wiki_term) == 0) ||
					  	  (strcmp($store_wiki_term, $last_wiki_term) != 0)) {
						//write the previous key list
						//if the list wiki term is not equal ot the last wiki term
						$last_wiki_term = $store_wiki_term;
						$h_val = writeBoBEntry($key_list,$syn_list,$last_alp,$phrase_list,$count_list);
						$txt .= $h_val[0];
						$last_alp = $h_val[1];
					  }
				  }
				  //add this key as a new key list
				  $key_list = array($key);
				  $syn_list = $vals;
				  $store_wiki_term = $new_wiki_term;//getWikitermFrom($syn_list);
			  }
			}
		}
		if (count($key_list) > 0) {
			if (strcmp($syn_list[0], $last_wiki_term) != 0) {
				//write the previous key list
				//if the list wiki term is not equal ot the last wiki term
				$h_val = writeBoBEntry($key_list,$syn_list,$last_alp,$phrase_list,$count_list);
				$txt .= $h_val[0];
			}
		}
		$txt .= "</bob><newpage /><newpage />\n";$time2 = now($stime);timeLog("writeBoB", $time2-$time1);
		
		return $txt;
	}
	
	/*
		Choose the correct capitalization for each term
		in the phrase list
		@param array $phrase_list the list of the term with its original capitalization
	*/
	function chooseTerm ($phrase_list) {
		$keys = array_keys($phrase_list);
		for ($i = 0; $i < count($keys); $i++) {
			$key = $keys[$i];
			
			$wiki_term = "";//the wiki term
			$rb_term = "";//chosen word from the book
			$rb_count = 0;//the number of uppercase in rb_term
			$val = $phrase_list[$key];
			for ($j = 0; $j < count($val); $j++) {
				$term = $val[$j];
				
				if (stripos($term, "wikiid:") !== false) {
					//for now just store them
					$wiki_term = substr($term, strlen("wikiid:"));
				} else {
					//choose the one that has the smallest number of capitals
					$t_count = countUppercase($term);
					if ((strlen($rb_term) == 0) || ($t_count < $rb_count)) {
						$rb_count = $t_count;
						$rb_term = $term;
					}
				}
			}

			$new_val = array();
			if (strlen($rb_term) > 0) {
				//assume that the version written in the book is always correct
				array_push($new_val, $rb_term);
			} else if (strlen($wiki_term) > 0) {
				//no reference from the book
				array_push($new_val, $wiki_term);
			}
			
			$phrase_list[$key] = $new_val;
		}
		
		return $phrase_list;
	}
	
	/*
		If the term count is less than 1, we apply the heuristic
		of making the word lowercase, if the capitalisation only in the start
		of the word
		@param string $str the term to be capitalised
		@param number $counter the number of time the term is mentioned in the book
	*/
	function applyCapital ($str, $counter) {
		if ((!$counter) || ($counter <= 1)) {
			//either we have only seen this term in the book once
			//or this term is a wiki term that does not appear in the book
			if (isStartCapital($str) == true) {
				if ($counter == 1) {
					//if the term is only a word that appears
					//once in the book
					if (countSpace($str) == 0)
						return $str;
				}
				$str = strtolower($str);
			}
		}
		
		return $str;
	}
	
	/*
		Write the string to both of the array entry
		or just to the first array entry
		@param array   $arr  	array that need to be modified
		@param boolean $isBoth 	whether the data should inserted to both array entry
		@param string  $str 	text to be inserted
	*/
	function appendToEntry ($arr, $isBoth, $str) {
		$arr[0] = $arr[0].$str;
		if ($isBoth == true) {
			$arr[1] = $arr[1].$str;
		}
		return $arr;
	}
	
	/*
		Write all the synonyms in the redirect list to the synonym file
		and create the back of the book index
		@param string $folder_name the path to the book directory
		@param array  $index_list [the list of redirects, the list of the phrases]
	*/
	function saveRedirects ($folder_name, $index_list) {$stime = microtime(); $time0 = now($stime);
		$redirect_list = $index_list[0];
		$count_list = $index_list[2];
		$index_list[1] = chooseTerm ($index_list[1]);
		$phrase_list = $index_list[1];
		
		$data_arr = array("","");
		
		$data_arr = appendToEntry($data_arr, true, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
		$data_arr = appendToEntry($data_arr, true, "<RedirectList>\n");
		$keys = array_keys($redirect_list);
		sort($keys);
		for ($i = 0; $i < count($keys); $i++) {
			$key = $keys[$i];
			
			$iswiki = false;
			$entry_data = "";
			//write entry in the synonym file
			$entry_data .= "\t<Redirect>\n";
			$entry_data .= "\t\t<Term title=\"".$key."\" />\n";
			
			sort($redirect_list[$key]);
			$val = $redirect_list[$key];
			for ($j = 0; $j < count($val); $j++) {
					$term = $val[$j];
					
					//write entry in the synonym file
					$entry_data .= "\t\t<Synonym title=\"".$term."\" ";
					if ((stripos($term, "wikiid:") !== false) || 
						(strcmp($term, "rb:true") == 0)){
							
						if (stripos($term, "wikiid:") !== false) {
							$iswiki = true;
						}
						
						$entry_data .= "phrase=\"".$phrase_list[$key][0]."\" ";
						$entry_data .= "count=\"";
						if ($count_list[$key]) {
							$entry_data .= $count_list[$key];
						} else {
							$entry_data .= "0";
						}
						$entry_data .= "\" ";
					}
					$entry_data .= "/>\n";
			}
			$entry_data .= "\t</Redirect>\n";
			$data_arr = appendToEntry($data_arr, $iswiki, $entry_data);
		}
		$data_arr = appendToEntry($data_arr, true, "</RedirectList>");
		
		$data = iconv('UTF-8', 'ISO-8859-1//IGNORE//TRANSLIT', $data_arr[0]);
		$syn_file = "repository/redirect.xml";
		$fh = modifyFile($folder_name.'/'.$syn_file, 'w');
		if (!$fh) die("can't open file");
		fwrite($fh, $data);
		fclose($fh);$time1 = now($stime);timeLog("saveSynonyms", $time1-$time0);
		
		$syn_data = iconv('UTF-8', 'ISO-8859-1//IGNORE//TRANSLIT', $data_arr[1]);
		$red_file = "synonym.xml";
		$fh = modifyFile($folder_name.'/'.$red_file,'w');
		if (!$fh) die("can't open file");
		fwrite($fh, $syn_data);
		fclose($fh);$time2 = now($stime);timeLog("saveRedirects", $time2-$time1);
		
		$txt = writeBoB ($redirect_list, $phrase_list, $count_list);$time3 = now($stime);timeLog("end saveRedirects", $time3-$time2);
		return $txt;
	}
	
	/*
		Given an entry, if its first word is either "a","an" or "the", remove
		the first word from the entry
		@param string $str the entry
	*/
	function cleanEntry($str) {
		$sArr = explode(" ",$str);
		$sArr[0] = strtolower($sArr[0]);
		$stop = array("a","an","the");
		if (in_array($sArr[0],$stop) == true){
			return substr($str, strlen($sArr[0]) + 1);
		}
		return $str;
	}
	
	/*
		Given a Wikipedia title, check that the given text in the book
		is the same with the title. If they are, check the capitalisation.
		If the capitalisation match, return true.
		@param string $wText 		Wikipedia title
		@param string $book_text	the text in the book 
	*/
	function isMatchWikiCapital ($wText, $book_text,$comment) {
		if (strlen($wText) == 0) {
			return false;
		}
		
		if ((strlen($comment) == 0) || (strcmp(strtolower($wText), strtolower($book_text)) == 0)) {
			if (strcmp($wText, $book_text) == 0) {//match exact text
				return true;
			} else if ((strlen($comment) == 0) || 
						(strcmp($comment, "a:") != 0)) {	
				$wUpper = countUppercase($wText);
				$tUpper = countUppercase($book_text);
				if ($wUpper == 0) {//no uppercase (--should never happen)
					return true;
				} else if (($wUpper > 1) && ($tUpper >= 1)) {
					//if the wiki has more than one uppercase, 
					//the text should have uppercase
					return true;
				} else if ($wUpper == 1) {
					if (isStartCapital($wText) == false) {
						//if the wiki has an uppercase not in the start
						//of the title, the book text should be the same as well
						$tUpper = countUppercase(substr($book_text, 1));
						if ($tUpper >= 1) {
							return true;
						}
					} else {
						//if it is only at the start, we are not sure
						//that the wiki article should have uppercase
						//or not, so just return true
						return true;
					}
				}
			}
		}
		return false;
	}
	
	/*
		If title for the given Wikipedia article has more than 1 capital,
		the given text in the book should have at least 1 capital. Otherwise,
		it is assume that the text has been linked incorrectly. It is assume
		that David's wikification rarely wrong. So if we cannot find the exact
		match for it, just check capitalisation based on main title for that
		article.
		@param int 		$wiki_id 	the wikipedia article id
		@param string 	$book_text	the text in the book
	*/
	function isCorrectWikiLink ($wiki_id, $book_text) {
		$url = $GLOBALS["wikiaddr"]."task=search&xml&id=".$wiki_id;
		$xmlContent = file_get_contents($url);
		$book_text = cleanEntry($book_text);
		
		//get the wikipedia article title
		preg_match_all("/<Article[\s]+([^>]+)>/", $xmlContent, $tmp_xmatches, PREG_SET_ORDER);
		foreach ($tmp_xmatches as $xmatches) {
			preg_match_all("/title=\"([^\"]+)\"/", $xmatches[1], $tmp_tmatches, PREG_SET_ORDER);
			foreach ($tmp_tmatches as $tmatches) {
				$wText = cleanEntry(cleanWikiTitle($tmatches[1]));
				if (isMatchWikiCapital($wText, $book_text,"") == true) {
					return isPerson($xmlContent, $book_text);
				}
				break;
			}
			break;
		}
		
		//get the wikipedia redirect
		preg_match_all("/<Redirect[\s]+([^>]+)>/", $xmlContent, $tmp_xmatches, PREG_SET_ORDER);
		foreach ($tmp_xmatches as $xmatches) {
			preg_match_all("/title=\"([^\"]+)\"/", $xmatches[1], $tmp_tmatches, PREG_SET_ORDER);
			foreach ($tmp_tmatches as $tmatches) {
				$wText = cleanEntry(cleanWikiTitle($tmatches[1]));
				if (isMatchWikiCapital($wText, $book_text,"r:") == true) {
					return isPerson($xmlContent, $book_text);
				}
			}
		}
		
		//get the wikipedia anchor
		preg_match_all("/<Anchor[\s]+([^>]+)>/", $xmlContent, $tmp_xmatches, PREG_SET_ORDER);
		foreach ($tmp_xmatches as $xmatches) {
			preg_match_all("/text=\"([^\"]+)\"/", $xmatches[1], $tmp_tmatches, PREG_SET_ORDER);
			foreach ($tmp_tmatches as $tmatches) {
				$wText = cleanEntry(cleanWikiTitle($tmatches[1]));
				if (isMatchWikiCapital($wText, $book_text,"a:") == true) {
					return isPerson($xmlContent, $book_text);
				}
			}
		}
		
		return "false";
	}
	
	/*
		Given a wikipedia article xml definition, see if it is a person name.
		We do this by simple testing of looking at redirect
		@param String $xmlContent the wikipedia article xml defintion
		@param String $book_text  the text in the book
	*/
	function isPerson ($xmlContent, $book_text) {
		//get the wikipedia article title
		preg_match_all("/<Article[\s]+([^>]+)>/", $xmlContent, $tmp_xmatches, PREG_SET_ORDER);
		foreach ($tmp_xmatches as $xmatches) {
			preg_match_all("/title=\"([^\"]+)\"/", $xmatches[1], $tmp_tmatches, PREG_SET_ORDER);
			foreach ($tmp_tmatches as $tmatches) {
				$wText = strtolower(cleanWikiTitle($tmatches[1]));
				break;
			}
			break;
		}
	
		//the string must not contain comma, if it does
		//the string is not converted
		if (stripos($wText,",") === false) {
			//get the last word
			$sp = strripos($wText," ");
			if ($sp !== false) {
				$lname = substr($wText, $sp + 1);
				$fname = substr($wText, 0, $sp);
				
				//see if this article has (Clarke, *) as its redirect or anchor
				preg_match_all("/<Redirect[\s]+([^>]+)>/", $xmlContent, $tmp_xmatches, PREG_SET_ORDER);
				foreach ($tmp_xmatches as $xmatches) {
					if (strripos($xmatches[1], "=\"".$lname.", ") !== false) {
						return "person";
					} 
				}
				preg_match_all("/<Anchor[\s]+([^>]+)>/", $xmlContent, $tmp_xmatches, PREG_SET_ORDER);
				foreach ($tmp_xmatches as $xmatches) {
					if (strripos($xmatches[1], "=\"".$lname.", ") !== false) {
						return "person";
					} 
				}
			}
		}
		
		//see if the article belongs to Living People, births or deaths category
		preg_match_all("/<Category[\s]+([^>]+)>/", $xmlContent, $tmp_xmatches, PREG_SET_ORDER);
		foreach ($tmp_xmatches as $xmatches) {
			if (strripos($xmatches[1], "title=\"Living People\"") !== false) {
				return "person";
			} else if (preg_match("/title=\"[0-9]+ births\"/i", $xmatches[1])) {
				return "person";
			} else if (preg_match("/title=\"[0-9]+ deaths\"/i", $xmatches[1])) {
				return "person";
			} 
		}
		
		//see if the first paragraph contain the word </b> (born, or </b> (date) or </b> (date - date) 
		$fStart = stripos($xmlContent,"<FirstParagraph>");
		if ($fStart !== false) {
			$fEnd = stripos($xmlContent,"</FirstParagraph>");
			if ($fEnd !== false) {
				$fStart = $fStart + strlen("<FirstParagraph>");
				$parText = substr($xmlContent, $fStart, $fEnd - $fStart);
				$parText = trim(strtolower($parText));
				$parText = str_replace("&amp;","&",$parText);
				$parText = str_replace("&ndash;","-",$parText);
				$parText = str_replace("&mdash;","-",$parText);
				$parText = strip_tags($parText, "<b>");
				if (preg_match("/<\/b>[\s]+([^a-z]*born[\s]+[^)]+)/i",$parText)) {
					return "person";
				} else {
					preg_match_all("/<\/b>[\s]+(\(([^)]+)\))/i",$parText, $tmp_xmatches, PREG_SET_ORDER);
					foreach ($tmp_xmatches as $xmatches) {
						if (isDate($xmatches[2]) == true) {
							return "person";
						} else {
							preg_match_all("/<\/b>[\s]+\(([a-z 0-9,]+)\-([a-z 0-9,]+)\)/i",$xmatches[1], 
											$tmp_tmatches, PREG_SET_ORDER);print_r($tmp_tmatches);
							foreach ($tmp_tmatches as $tmatches) {
								if ((isDate($tmatches[1])) || (isDate($tmatches[2]))) {
									return "person";
								}
							}
						}
					}
				}
			}
		}
		
		return "true";
	}
	
	/*
		Check if the given string is a date
		@param string $str the text to be tested
	*/
	function isDate($str){
		$stamp = strtotime( $str );
	   
		if (!is_numeric($stamp)){
		   return false;
		}
		$month = date( 'm', $stamp );
		$day   = date( 'd', $stamp );
		$year  = date( 'Y', $stamp );
	   
		if (checkdate($month, $day, $year)){
		   return true;
		}
	   
		return false;
	} 
	
	/*
		Convert person name from Arthur C. Clark to Clark, Arthur C.
		@param string $str person name
	*/
	function convertName ($str) {
		//the string must not contain comma, if it does
		//the string is not converted
		if (stripos($str,",") === false) {
			//get the last word
			$sp = strripos($str," ");
			if ($sp !== false) {
				$lname = substr($str, $sp + 1);
				$fname = substr($str, 0, $sp);
				return trim($lname.", ".$fname);
			}
		}
		return $str;
	}
	
	/*
		Count the number of spaces in the term
		@param string $str the term
	*/
	function countSpace($str) {
		preg_match_all("/\s/",$str, $matches);
		return count($matches[0]);
	}
	
	/*
	 	Count the number of uppercase in the term
		@param string $str the term
	*/
	function countUppercase($str){ 
		preg_match_all("/[A-Z]/",$str,$matches); 
		return count($matches[0]); 
	}
	
	/*
		Test whether the term only has one capitalization
		and that is at the beginning of the word
		@param string $str the term
	 */
	function isStartCapital($str) {
		$count_upper = countUppercase($str);
		if ($count_upper == 1) {
			$first_char = substr($str, 0, 1);
			$second_char = substr($str, 1, 1);
			if ((countUppercase($first_char) == 1) &&
				(strcmp($second_char, " ") != 0)) {
				return true;
			}
		}
		return false;
	}
	
	/*
		Removes bracket from a string and remove string that is not 
		compatible with ISO-8859-1.
		@param string $str the string
	*/
	function cleanWikiTitle ($str) {
		$str = iconv('UTF-8', 'ISO-8859-1//IGNORE//TRANSLIT', $str);
	  $nArr = array ("(",")","\"");
		$str = str_replace($nArr, "", $str);
		$str = str_replace("<", "&lt;", $str);
		$str = str_replace(">", "&gt;", $str);
		$str = str_replace("&", "&#38;", $str);
		
		/*if (preg_match("/^[\\x00-\\xFF]*$/u", $str) === 1) {
			
		} else {
			$str = "";
		}*/
		
		return $str;
	}
	
	/*
		Removes the word 'the' from the term
		@param string $str the string
	*/
	function cleanIndexEntry ($t_str) {
		$t_str = cleanWikiTitle($t_str);
		$t_str = cleanEntry($t_str);
		return $t_str;
	}
	
	/*
		Removes the word 'the' from the redirect term, and perform
		the same cleaning as for the wiki article
		@param string $str the string
	*/
	function cleanRedirectTitle ($str) {
		$str = cleanWikiTitle(strtolower($str));
		return $str;
	}
	
	/*
		Set the specified text as a redirect to the specified article
		@param string $aTitle the article title
		@param string $aText the redirects
		@param array  $redirect_list the list of synonyms
		@param number $stat whether the redirect is found in the book
	 */
	function setTextAsRedirect ($aTitle, $aText, $redirect_list, $stat) {
		$aText = cleanRedirectTitle($aText);
		
		if (strlen($aText) == 0)
			return $redirect_list;
		
		if ((strcmp($aText, $aTitle) != 0) && (!in_array($aText, $redirect_list[$aTitle]))) {
				array_push($redirect_list[$aTitle], $aText);
				if (!$redirect_list[$aText]) {
					$redirect_list[$aText] = array();
				}
				if (!in_array($aTitle, $redirect_list[$aText])) {
					array_push($redirect_list[$aText], $aTitle);
				}
				
				if (count($redirect_list[$aText]) > 1)
					echo "syn already exist for $aText when parsing $aTitle called from $stat <br />";
		}
		
		if (($stat == 1) && (!in_array("rb:true", $redirect_list[$aText]))) {
			array_push($redirect_list[$aText], "rb:true");
		}
		
		return $redirect_list;
	}
	
	/*
		Get all the redirects for the specified article and enter it to the redirect list
		@param string $aTitle the article title
		@param array $redirect_list the list of redirects in the documents
		@param string $aID the article id
	*/
	function getRedirects ($aTitle, $redirect_list, $aID) {
		if (!$redirect_list[$aTitle]) {
			$redirect_list[$aTitle] = array();
			array_push($redirect_list[$aTitle],"wikiid:".$aID);
		}
		
		$synurl = $GLOBALS["wikiaddr"]."task=search&xml&id=".$aID;
		$synxml = file_get_contents($synurl);
		//get the list of redirects
		preg_match_all("/<Redirect[\s]+([^>]+)>/", $synxml, $tmp_rmatches, PREG_SET_ORDER);
		foreach ($tmp_rmatches as $rmatches) { 
			preg_match_all("/title=\"([^\"]+)\"/", $rmatches[1], $tmp_tmatches, PREG_SET_ORDER);
			foreach ($tmp_tmatches as $tmatches) {
				 $redirect_list = setTextAsRedirect($aTitle, $tmatches[1], $redirect_list, 0);
			}
		}
		
		return $redirect_list;
	}
?>